# -*- coding: utf-8 -*-
import requests
from lxml import etree
import re
import traceback
import time
from loguru import logger
import random
import datetime
import execjs
import threading
import sys
import os
import json
from queue import Queue, Empty
from fake_useragent import UserAgent
import concurrent.futures
from src.service.requests_middler import requests_sys
from src.service.requests_middler import proxy_requests_sys
from src.service.requests_middler import insert_shipping_info
from src.service.requests_middler import read_excel
from src.service.requests_middler import read_transport_json
from src.service.requests_middler import get_proxies
ip_queue = Queue()

ua = UserAgent()
random_user_agent = ua.random
stop_threading_event = threading.Event()
current_working_directory = os.getcwd()
total_requests = 0
loss_requests = 0

def proxies_queue():
    # 获取代理并放入队列
    if ip_queue.empty():
        proxy = get_proxies()
        logger.info(f"获取ip代理成功: {proxy}")
        if proxy:
            ip_queue.put(proxy)
        else:
            logger.error(f"获取ip代理失败")
    else:
        proxy = ip_queue.get()
        return proxy

def msc_transport(origin_city, destination_city):
    try:
        global total_requests
        global loss_requests
        port_mapping = {}
        file_path = "msc_maeu_prots.json"
        # # file_path = os.path.join(current_working_directory + r'/src/crawler/msc', file_path)
        # file_path = os.path.join(current_working_directory, file_path)
        file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), file_path)
        if os.path.exists(file_path):
            with open(file_path, "r") as file:
                Menu_json = json.load(file)
        else:
            logger.error(f"不存在文件: {file_path}")
            sys.exit()
        for info in Menu_json['Ports']:
            port_mapping[info['LocationName'].upper()] = {
                'portCode': info['LocationName'],
                'unLocationCode': info['LocationCode'],
                'PortId': info['PortId']
            }
        if origin_city.upper() == "TIANJIN":
            origin_city = "TIANJINXINGANG"
        origin_info = port_mapping.get(origin_city.upper())
        destination_info = port_mapping.get(destination_city.upper())
        if origin_info and destination_info:
            total_requests += 1
            date = datetime.datetime.now().strftime('%Y-%m-%d')
            origin_unLocationCode = origin_info['unLocationCode']
            des_unLocationCode = destination_info['unLocationCode']
            fromPortId = origin_info['PortId']
            toPortId = destination_info['PortId']
            headers = {
                "accept": "application/json, text/plain, */*",
                "accept-language": "zh-CN,zh;q=0.9",
                "cache-control": "no-cache",
                "content-type": "application/json",
                "origin": "https://www.msccargo.cn",
                "pragma": "no-cache",
                "priority": "u=1, i",
                "referer": "https://www.msccargo.cn/en/search-a-schedule",
                "sec-ch-ua": "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"",
                "sec-ch-ua-mobile": "?0",
                "sec-ch-ua-platform": "\"Windows\"",
                "sec-fetch-dest": "empty",
                "sec-fetch-mode": "cors",
                "sec-fetch-site": "same-origin",
                "user-agent": random_user_agent,
                "x-requested-with": "XMLHttpRequest"
            }
            url = "https://www.msccargo.cn/api/feature/tools/SearchSailingRoutes"
            data = {
                "FromDate": f"{date}",
                "fromPortId": str(fromPortId),
                "toPortId": str(toPortId),
                "language": "en",
                "dataSourceId": "{E9CCBD25-6FBA-4C5C-85F6-FC4F9E5A931F}"
            }
            data = json.dumps(data, separators=(',', ':'))
            retry_count = 0
            while retry_count < 3:
                proxies = proxies_queue()
                # res = requests_sys(url, headers=headers, data=data, POST=True)
                res = proxy_requests_sys(url, headers=headers, data=data, POST=True, proxies=proxies)
                if res:
                    js_data = res.json()
                    logger.info(f"当前查询 起点:{origin_city} ----> 终点: {destination_city}")
                    logger.info(f"当前请求总数: {total_requests}, 失败请求总数: {loss_requests}")
                    if js_data.get("Data"):
                        data = js_data["Data"]
                        if isinstance(data, list) and len(data) > 0:
                            ip_queue.put(proxies)
                            for info in js_data['Data']:
                                for info2 in info['Routes']:
                                    item = {}
                                    if len(info2['RouteScheduleLegDetails']) >= 2:
                                        logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 存在中转站")
                                        item['origin_code'] = f"{info2['RouteScheduleLegDetails'][0]['DeparturePortName']}({info2['RouteScheduleLegDetails'][0]['DeparturePortUNCode']})"
                                        if "TIANJINXINGANG" in item['origin_code']:
                                            item['origin_code'] = "TIANJIN(CNTJN)"
                                        country_code = check_country_code(
                                            info2['RouteScheduleLegDetails'][-1]['ArrivalPortUNCode'])
                                        item['destination_code'] = (
                                            f"{info2['RouteScheduleLegDetails'][-1]['ArrivalPortName']}"
                                            f",{country_code}({info2['RouteScheduleLegDetails'][-1]['ArrivalPortUNCode']})")  # check

                                        item['depart_date'] = datetime.datetime.strptime(
                                            info2["EstimatedDepartureDate"], "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d")
                                        item['arrive_date'] = datetime.datetime.strptime(info2["EstimatedArrivalDate"],
                                                                                         "%Y-%m-%dT%H:%M:%S").strftime(
                                            "%Y-%m-%d")
                                        item['transport_contractor'] = "MSC"
                                        # departure_date = datetime.datetime.strptime(item['depart_date'], '%Y-%m-%d')
                                        # arrival_date = datetime.datetime.strptime(item['arrive_date'], '%Y-%m-%d')
                                        # days_difference = (arrival_date - departure_date).days
                                        item['haulage_time'] = info2['TotalTransitTime'].split("days")[0]
                                        item['ship_name'] = info2["VesselName"]
                                        item['voyage_number'] = info2["DepartureVoyageNo"]
                                        item['transfer_code'] = None
                                        item['shipping_lanes '] = shipping_lanes if (shipping_lanes := info2["RouteScheduleLegDetails"][0]["MaritimeServiceName"].split('SERVICE')[0].strip()) else None
                                        print(item)
                                        insert_shipping_info(item)

                                    else:
                                        item[
                                            'origin_code'] = f"{info2['RouteScheduleLegDetails'][0]['DeparturePortName']}({info2['RouteScheduleLegDetails'][0]['DeparturePortUNCode']})"
                                        if "TIANJINXINGANG" in item['origin_code']:
                                            item['origin_code'] = "TIANJIN(CNTJN)"
                                        country_code = check_country_code(
                                            info2['RouteScheduleLegDetails'][-1]['ArrivalPortUNCode'])
                                        item['destination_code'] = (
                                            f"{info2['RouteScheduleLegDetails'][-1]['ArrivalPortName']}"
                                            f",{country_code}({info2['RouteScheduleLegDetails'][-1]['ArrivalPortUNCode']})")  # check

                                        item['depart_date'] = datetime.datetime.strptime(
                                            info2["EstimatedDepartureDate"], "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d")
                                        item['arrive_date'] = datetime.datetime.strptime(info2["EstimatedArrivalDate"],
                                                                                         "%Y-%m-%dT%H:%M:%S").strftime(
                                            "%Y-%m-%d")
                                        item['transport_contractor'] = "MSC"
                                        item['haulage_time'] = info2['TotalTransitTime'].split("days")[0]
                                        item['ship_name'] = info2["VesselName"]
                                        item['voyage_number'] = info2["DepartureVoyageNo"]
                                        item['transfer_code'] = None
                                        print(item)
                                        insert_shipping_info(item)

                                    if info2['RouteScheduleLegDetails'][-1]['ArrivalPortName'] is None:
                                        logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 为空")
                                        continue
                        else:
                            logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 获取数据失败")
                            read_transport_json(transport='MSC', origin_city=origin_city, des_city=destination_city,
                                           fail_result="获取数据失败")
                            loss_requests += 1
                            retry_count += 1
                    else:
                        logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 获取数据失败")
                        read_transport_json(transport='MSC', origin_city=origin_city, des_city=destination_city,
                                       fail_result="获取数据失败")
                        loss_requests += 1
                        retry_count += 1
                    break
                elif res == 0:
                    loss_requests += 1
                    logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 获取数据失败, 请求超时")
                    read_transport_json(transport='MSC', origin_city=origin_city, des_city=destination_city,
                                   fail_result="请求超时")
                    retry_count += 1
    except Exception as e:
        logger.error(f"{origin_city} ----> {destination_city}: {e}")

def check_code(code):
    # file = 'maeu_ports.json'
    # file = os.path.join(current_working_directory + r'/src/crawler', file)
    # file = os.path.join(os.path.dirname(current_working_directory), 'maeu_ports.json')
    file_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'maeu_ports.json')

    with open(file_path, 'r') as f:
        result = f.read()
        f.close()
    null = ''
    true = ''
    false = ''
    result = eval(result)['ports']
    for res in result:
        if code == res['unLocationCode'] and (code[0:2] == 'US' or code[0:2] == 'CA'):
            return True
    return False


def check_country_code(code):
    # file = 'maeu_ports.json'
    # file = os.path.join(current_working_directory + r'/src/crawler', file)
    # file = os.path.join(os.path.dirname(current_working_directory), 'maeu_ports.json')
    file_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'maeu_ports.json')

    with open(file_path, 'r') as f:
        result = f.read()
        f.close()
    null = ''
    true = ''
    false = ''
    result = eval(result)['ports']
    for res in result:
        if code == res['unLocationCode'] and (code[0:2] == 'US' or code[0:2] == 'CA'):
            try:
                code = res['regionCode']
                return code
            except:
                return ''
    return ''

#MSC
def process_route(origin_city, destination_list):
    for destination_city in destination_list:
        msc_transport(origin_city, destination_city)

def msc_spider_run():
    global total_requests
    global loss_requests
    # origin_city_list = [
    #     "Shanghai", "XIAMEN", "NANSHA", "QINGDAO", "YANTIAN", "NINGBO"
    # ]
    #
    #
    # destination_list = [
    #     "Los Angeles", "Long Beach", "Atlanta", "Baltimore", "Boston", "Buffalo", "Calgary", "Charleston",
    #     "Chicago", "Chippewa Falls", "Cincinnati", "Cleveland", "Dallas", "Denver",
    #     "Detroit", "Edmonton", "EL Paso", "Halifax", "Houston", "Jacksonville",
    #     "Kansas City", "Louisville", "Memphis", "Miami", "Minneapolis", "Montreal",
    #     "Nashville", "New Orleans", "New York", "Norfolk", "Oakland", "Omaha",
    #     "Philadelphia", "Pittsburgh", "Portland", "Prince Rupert", "Saint Louis",
    #     "Salt Lake City", "San Antonio", "Saskatoon", "Savannah", "Seattle",
    #     "Tacoma", "Tampa", "Toronto", "Vancouver", "Winnipeg"
    # ]


    # with concurrent.futures.ThreadPoolExecutor() as executor:
    #     futures = [executor.submit(process_route, origin_city, destination_list) for origin_city in origin_city_list]

    json_path_one = r"origin_and_destination.json"
    # json_path_one = r"origin_and_destination - 副本.json"
    json_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), json_path_one)
    with open(f'{json_path}', 'r', encoding='utf-8') as f:
        tmp = json.loads(f.read())

    origin_city_list = tmp['origin_city_list']
    destination_list = tmp['destination_list']
    del tmp, json_path, json_path_one

    with concurrent.futures.ThreadPoolExecutor() as executor:
        futures = []
        for origin_city in origin_city_list:
            future = executor.submit(process_route, origin_city, destination_list)
            futures.append(future)
            time.sleep(5)

        for future in concurrent.futures.as_completed(futures):
            try:
                future.result()
            except Exception as e:
                # 输出异常详细信息，包括出错的行号
                print(f"An error occurred: {e}")
                print(traceback.format_exc())
    read_excel(transport="MSC", total_requests=total_requests, loss_requests=loss_requests)

if __name__ == '__main__':
    msc_spider_run()